package cn.edu.hznu.web.crawl;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Stream;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class CrawlHeadPics {

	final static String RESOURCE_FILE_PATH = "/Users/LVZY/Downloads/QQ/user.txt";
	final static String STORAGE_DIR = "/Users/LVZY/Documents/Record/Crawl/lianlian/";

	public void crawl() {

		LinkedList<String[]> lstLine = new LinkedList<String[]>();

		try (BufferedReader bfReader = new BufferedReader(new FileReader(RESOURCE_FILE_PATH))) {
			String data = null;
			while ((data = bfReader.readLine()) != null) {
				String[] tmp = new String[2];
				tmp[0] = data.substring(0, data.indexOf(","));
				tmp[1] = data.substring(data.lastIndexOf(",") + 1, data.length());
				if (tmp[1].equals("\"\"")) {
					continue;
				}
				lstLine.add(tmp);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(8);
		for (int i = 0; i < 8; i++) {
			int index = i;
			executor.execute(() -> {

				System.out.println("Turn to thread -----> " + index);

				List<String[]> subList;
				try {
					subList = lstLine.subList(index * 10000, (index + 1) * 10000);
				} catch (IndexOutOfBoundsException e) {
					subList = lstLine.subList(index * 10000, lstLine.size());
				}

				try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
					File folder = new File(STORAGE_DIR + String.valueOf(index) + "/");
					if (!folder.exists()) {
						folder.mkdirs();
					}
					subList.forEach(item -> {
						System.out.println("Crawl ----> " + item[0] + "=====" + item[1]);

						try (BufferedOutputStream bfOutStream = 
								new BufferedOutputStream(
										new FileOutputStream(
												new File(folder, item[0] + ".png")))) {
							DownloadContent.downloadImgFromUrl(httpClient, item[1], bfOutStream);
						} catch (IOException e) {
							e.printStackTrace();
						}
					});
				} catch (IOException e) {
					e.printStackTrace();
				}
			});
		}
		executor.shutdown();
	}
	
	public void deleteVoidImg() {
//		Paths.get(STORAGE_DIR).forEach(folder -> {
//			if (folder.toFile().isDirectory()) {
//				System.out.println("fsph");
//			}
//		});;
		
		File imgFolder = new File(STORAGE_DIR);
		Arrays.stream(imgFolder.listFiles())
			  .skip(1)
			  .forEach(subFolder -> {
				  Arrays.stream(subFolder.listFiles())
				  		.parallel()
				  		.forEach(imgFile -> {
				  			System.out.println(this.bytes2kb(imgFile.length()));
				  			if (this.bytes2kb(imgFile.length()) == 4) {
				  				imgFile.delete();
				  			}
				  		});
			  });
	}
	
	private byte[] longToByte(long x) {
		ByteBuffer byteBf = ByteBuffer.allocate(Long.BYTES);
		byteBf.putLong(x);
		return byteBf.array();
	}
	
	private int bytes2kb(long bytes) {
		BigDecimal filesize = new BigDecimal(bytes);
//		BigDecimal megabyte = new BigDecimal(1024 * 1024);
		float returnValue;
//		returnValue = filesize.divide(megabyte, 2, BigDecimal.ROUND_UP).floatValue();
//		if (returnValue > 1)
//			return (returnValue + "  MB ");
		BigDecimal kilobyte = new BigDecimal(1024);
		returnValue = filesize.divide(kilobyte, 2, BigDecimal.ROUND_UP).floatValue();
		return (int)returnValue;
//		return (returnValue + "  KB ");
	}

	class RecursiveFileLineReader {
		/**
		 * 取得所有指定扩展名的文件的非空行
		 * 
		 * @param path
		 *            the path to start recursion
		 * @param extension
		 *            the file extension
		 * @return list of lines
		 */
		public List<String> readAllLineFromAllFilesRecursively(final String path,
				final String extension) {
			final List<String> lines = new ArrayList<>();
			// 对于所有的文件
			try (final Stream<Path> pathStream = Files.walk(Paths.get(path),
					FileVisitOption.FOLLOW_LINKS)) {
				// 过滤出文件以及给定的扩展名
				pathStream
						.filter((p) -> !p.toFile().isDirectory()
								&& p.toFile().getAbsolutePath().endsWith(extension))
						// 对每一个文件 读取每行的内容
						.forEach(p -> fileLinesToList(p, lines));
			} catch (final IOException e) {
				e.printStackTrace();
			}
			return lines;
		}

		private void fileLinesToList(final Path file, final List<String> lines) {
			try (Stream<String> stream = Files.lines(file, Charset.defaultCharset())) {
				// 只读取非空行，加入到lines中
				stream.map(String::trim).filter(s -> !s.isEmpty()).forEach(lines::add);
			} catch (final IOException e) {
				e.printStackTrace();
			}
		}
	}
	
}
